
##################################################################
##################################################################
## Replication Material
## Widmann & Wich: Creating and Comparing Dictionary, Word Embedding, and Transformer-based 
## Models to Measure Discrete Emotions in German Political Text
## Political Analysis
## tobias.widmann@eui.eu
##
## Script 02: Replication of Appendices
##################################################################
##################################################################

# Note: The file 000_readme.pdf describes all scripts and datasets required to replicate the analysis

# This script was run on the following R version, platform and OS:
# R version 4.0.5 (2021-03-31)
# Platform: x86_64-apple-darwin17.0 (64-bit)
# Running under: macOS Big Sur 11.5.1

sessionInfo()

#### Set Working Directory to the Replication Folder# ###########################

# Delete hashtag below and fill in the directory of the replication folder
# setwd("")

#### Load Packages ##############################################################

library("openxlsx")       # Version 4.2.3
library("irr")            # Version 0.84.1
library("stats")          # Version 4.1.1
library("quanteda")       # Version 3.0.0
library("reticulate")     # Version 1.21
library("corpus")         # Version 0.10.1
library("keras")          # Version 2.6.0
py_install("pandas")
library("randomForest")   # Version 4.6-14
library("glmnet")         # Version 4.1-1
library("e1071")          # Version 1.7-6
library("ROCit")          # Version 2.1.1
library("dplyr")          # Version 1.0.5
library("ggpubr")         # Version 0.4.0
library("ggplot2")        # Version 3.3.3
library("Rmisc")          # Version 1.5
library("dotwhisker")     # Version 0.6.0
library("stringi")        # Version 1.5.3
library("reshape2")       # Version 1.4.4
library("readr")          # Version 1.4.0
library("stargazer")      # Version 5.2.2



#### Appendix A ###########################################################

##### Load Data #################################################################
load("./coder1.Rdata")
load("./coder2.Rdata")


##### Table A.3 #################################################################
# To calculate Krippendorff Alphas, we first have to combine the results of coder 1 and coder 2
# in the loop. We then bring the individual coding results from each coder for each emotion together
# Then we calculate the Alpha score using the kripp.alpha function
# Results for each emotion are saved in the dataframe tableA3

coder1$name <- "coder1"
coder2$name <- "coder2"

score <- rbind(coder1, coder2)

tableA3 <- NULL
df_temp <- NULL

for (i in 2:9) {
  df1 <- score[,c(1,i,10)]
  df2 <- reshape(df1, idvar = "name", timevar = "feature", direction = "wide")
  df3 <- df2[,-1]
  rownames(df3) <- df2[,1]
  df3 <- as.matrix(df3)
  
  result <- kripp.alpha(df3)
  kripp_alpha <- result$value
  emotion <- colnames(score)[i]
  
  df_temp <- data.frame(emotion, kripp_alpha)
  tableA3 <- rbind(tableA3, df_temp)
}

print(tableA3)

# Exporting table to txt
stargazer(tableA3, summary = FALSE, out = "./tables/tableA3.txt")

#### Appendix D #############################################################
##### Load Data #################################################################
# To measure correlations in Appendix D, we rely on dataset 2, which was not pre-sampled.
# This dataset therefore represents a more realistic snapshot of political text data compared to dataset 1

load("./data2_prepared.Rdata")


##### Table D.1 #################################################################
# We count the occurrences of each emotion, based on human coding 

tableD1 <- data.frame(Emotion=c("anger", "fear", "disgust", "sadness", "joy",
                                "enthusiasm", "pride", "hope"), 
                      Occurrences=c(sum(data2_prepared$hf_anger), sum(data2_prepared$hf_fear),
                                   sum(data2_prepared$hf_disgust), sum(data2_prepared$hf_sadness),
                                   sum(data2_prepared$hf_joy), sum(data2_prepared$hf_enthusiasm),
                                   sum(data2_prepared$hf_pride), sum(data2_prepared$hf_hope)))

print(tableD1)

# Exporting table to txt
stargazer(tableD1, summary = FALSE, out = "./tables/tableD1.txt")

##### Table D.2 #################################################################
# We count the occurences of each emotion, seperated by source

tableD2 <- data.frame(Emotion=c("anger", "fear", "disgust", "sadness", "joy",
                                "enthusiasm", "pride", "hope"), 
                      Facebook=c(sum(data2_prepared$hf_anger[data2_prepared$type=="Facebook"]), sum(data2_prepared$hf_fear[data2_prepared$type=="Facebook"]),
                                 sum(data2_prepared$hf_disgust[data2_prepared$type=="Facebook"]), sum(data2_prepared$hf_sadness[data2_prepared$type=="Facebook"]),
                                 sum(data2_prepared$hf_joy[data2_prepared$type=="Facebook"]),sum(data2_prepared$hf_enthusiasm[data2_prepared$type=="Facebook"]),
                                 sum(data2_prepared$hf_pride[data2_prepared$type=="Facebook"]), sum(data2_prepared$hf_hope[data2_prepared$type=="Facebook"])),
                      Parliament=c(sum(data2_prepared$hf_anger[data2_prepared$type=="Parliament"]), sum(data2_prepared$hf_fear[data2_prepared$type=="Parliament"]),
                                   sum(data2_prepared$hf_disgust[data2_prepared$type=="Parliament"]), sum(data2_prepared$hf_sadness[data2_prepared$type=="Parliament"]),
                                   sum(data2_prepared$hf_joy[data2_prepared$type=="Parliament"]),sum(data2_prepared$hf_enthusiasm[data2_prepared$type=="Parliament"]),
                                   sum(data2_prepared$hf_pride[data2_prepared$type=="Parliament"]), sum(data2_prepared$hf_hope[data2_prepared$type=="Parliament"])))
              

print(tableD2)

# Exporting table to txt
stargazer(tableD2, summary = FALSE, out = "./tables/tableD2.txt")


##### Table D.3 #################################################################
# Correlations between emotions (ed8)

tabled3_data <- data2_prepared[, c("anger.norm", "fear.norm", "disgust.norm", "sadness.norm", "joy.norm", "enthusiasm.norm", "pride.norm", "hope.norm")]
res <- cor(tabled3_data)
tableD3 <- round(res, 3)
print(tableD3)

# Exporting table to txt
stargazer(tableD3, summary = FALSE, out = "./tables/tableD3.txt")


##### Table D.4 #################################################################
# Correlations between emotions (word embeddings)

tabled4_data <- data2_prepared[, c('wb.anger','wb.fear', 'wb.disgust', 'wb.sadness', 'wb.joy', 'wb.enthusiasm', 'wb.pride', 'wb.hope')]
res <- cor(tabled4_data)
tableD4 <- round(res, 3)
print(tableD4)

# Exporting table to txt
stargazer(tableD4, summary = FALSE, out = "./tables/tableD4.txt")

##### Table D.5 #################################################################
# Correlations between emotions (ELECTRA)

tabled5_data <- data2_prepared[, c("el_anger", "el_fear", "el_disgust", "el_sadness", "el_joy", "el_enthusiasm", "el_pride", "el_hope")]
res <- cor(tabled5_data)
tableD5 <- round(res, 3)
print(tableD5)

# Exporting table to txt
stargazer(tableD5, summary = FALSE, out = "./tables/tableD5.txt")


##### Table D.6 #################################################################
# Correlations between emotions (Human judgement)

tabled6_data <- data2_prepared[,c("hf_anger", "hf_fear", "hf_disgust", "hf_sadness", "hf_joy", "hf_enthusiasm", "hf_pride", "hf_hope")]
res <- cor(tabled6_data)
tableD6 <- round(res, 3)
print(tableD6)

# Exporting table to txt
stargazer(tableD6, summary = FALSE, out = "./tables/tableD6.txt")


#### Appendix E ##############################################################
##### Load Data #################################################################
load("./data1_prepared.Rdata")


##### Create Training and Test Data ############################################

set.seed(1111)
sample_size <- floor(.90 * nrow(data1_prepared))
train_ind <- sample(nrow(data1_prepared), size = sample_size)

test_data <- data1_prepared[-train_ind,]
training_data <- data1_prepared[train_ind,]



## function to compute accuracy
acc_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(sum(diag(tab))/sum(tab))
}
# function to compute precision
prec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return((tab[2,2])/(tab[2,1]+tab[2,2]))
}
# function to compute recall
rec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(tab[2,2]/(tab[1,2]+tab[2,2]))
}



##### Table E1: Polyglot ##################################################################

# To apply the 'simple' neural networks classifiers based on Polyglot word embeddings, we firstly create
# a corpus from our 10,000 sentences
cgcorpus <- corpus(data1_prepared$Text)

# Create a document feature matrix and conduct pre-processing
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

# Now, we first load in the Polyglot word embeddings which can be downloaded at "https://sites.google.com/site/rmyeid/projects/polyglot?authuser=0"

source_python("./read_pickle.py")
pickle_data <- read_pickle_file("./polyglot-de.pkl")

# Then we convert the word embeddings to a data frame, and then we will 

polyglot <- data.frame(pickle_data[2][[1]])
wordlist <- as.list(pickle_data[[1]])
polyglot$word <- wordlist
polyglot <- polyglot[,c(65,1:64)]

# Then we match the features from each document with their corresponding embeddings
# To increase matches we stem them first

polyglot$word <- text_tokens(polyglot$word, stemmer = "de")

w2v2 <- polyglot[polyglot$word %in% featnames(cgdfm),]

# creating new feature matrix for embeddings
embed_poly <- matrix(NA, nrow=ndoc(cgdfm), ncol=64)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embeddings for those words
  embed_vec <- w2v2[w2v2$word %in% doc_words, 2:65]
  # aggregate from word- to document-level embeddings by taking AVG
  embed_poly[i,] <- colMeans(embed_vec, na.rm=TRUE)
  # if no words in embeddings, simply set to 0
  if (nrow(embed_vec)==0) embed_poly[i,] <- 0
}

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion
#Note: the code for training of the models can be found in the file 03_training_polyglot.R

tableE1 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_poly[-train_ind,]
  y_test <- to_categorical(data1_prepared[paste0("hf_",i)][-train_ind,])
  model <- load_model_hdf5(paste0("./poly_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_poly[-train_ind,])
  acc_fun(predictions, data1_prepared[paste0("hf_",i)][-train_ind,])
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableE1 <- rbind(tableE1, df_temp)
}

round(tableE1[,-1], digits=2)

# Exporting table to txt
stargazer(tableE1, summary = FALSE, out = "./tables/tableE1.txt", digits = 2)


##### Table E2: Bojanowski et al. ##################################################

# Now, we replace the locally word embeddings with embeddings by Bojanowski et al. (2017)
# These can be downloaded at "https://fasttext.cc/docs/en/pretrained-vectors.html"
# To apply the 'simple' neural networks classifiers based on word embeddings, we firstly create
# a corpus from our 10,000 sentences
cgcorpus <- corpus(data1_prepared$Text)

# Create a document feature matrix and conduct pre-processing
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

# Load in the word embeddings
# Note: this step takes a very long time, but it can be skipped by 
# loading the file "embed_boja.Rdata" below

lines <- readLines("./boja_wiki.de.vec")

# Turn into data frame
df1 <- read.table(text=lines, sep="", quote="", fill = TRUE)
# Rename first column
colnames(df1)[1] <- "word"

# Stemming
df1$word <- text_tokens(df1$word, stemmer = "de")

# Matching
w2v2 <- df1[df1$word %in% featnames(cgdfm),]

# creating new feature matrix for embeddings
embed_boja <- matrix(NA, nrow=ndoc(cgdfm), ncol=300)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embeddings for those words
  embed_vec <- w2v2[w2v2$word %in% doc_words, 2:301]
  # aggregate from word- to document-level embeddings by taking AVG
  embed_boja[i,] <- colMeans(embed_vec, na.rm=TRUE)
  # if no words in embeddings, simply set to 0
  if (nrow(embed_vec)==0) embed_boja[i,] <- 0
}

load("./embed_boja.Rdata")

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion
#Note: the code for training of the models can be found in the file 03_training_bojanowski.R

tableE2 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_boja[-train_ind,]
  y_test <- to_categorical(data1_prepared[paste0("hf_",i)][-train_ind,])
  model <- load_model_hdf5(paste0("./boja_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_boja[-train_ind,])
  acc_fun(predictions, data1_prepared[paste0("hf_",i)][-train_ind,])
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableE2 <- rbind(tableE2, df_temp)
}


round(tableE2[,-1], digits=2)

# Exporting table to txt
stargazer(tableE2, summary = FALSE, out = "./tables/tableE2.txt", digits = 2)

##### Table E3: Mikolov et al. 2017 ##################################################

# Now, we replace the locally word embeddings with embeddings by Mikolow et al. (2017)
# These can be downloaded at "https://fasttext.cc/docs/en/crawl-vectors.html"
# To apply the 'simple' neural networks classifiers based on word embeddings, we firstly create
# a corpus from our 10,000 sentences
cgcorpus <- corpus(data1_prepared$Text)

# Create a document feature matrix and conduct pre-processing
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

# Load in embeddings
# Note: this step takes a very long time, but it can be skipped by 
# loading the file "embed_miko.Rdata" below
lines <- readLines("./miko_cc.de.300.vec")

df1 <- read.table(text=lines, sep="", quote="", fill = TRUE, skip = 1)
head(df1)
colnames(df1)[1] <- "word"

# Stem embeddings
df1$word <- text_tokens(df1$word, stemmer = "de")

# Match embeddings with corpus
w2v2 <- df1[df1$word %in% featnames(cgdfm),]

# creating new feature matrix for embeddings
embed_miko <- matrix(NA, nrow=ndoc(cgdfm), ncol=300)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embeddings for those words
  embed_vec <- w2v2[w2v2$word %in% doc_words, 2:301]
  # aggregate from word- to document-level embeddings by taking AVG
  embed_miko[i,] <- colMeans(embed_vec, na.rm=TRUE)
  # if no words in embeddings, simply set to 0
  if (nrow(embed_vec)==0) embed_miko[i,] <- 0
}

load("./embed_miko.Rdata")

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion
#Note: the code for training of the models can be found in the file 03_training_mikolov.R

tableE3 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_miko[-train_ind,]
  y_test <- to_categorical(data1_prepared[paste0("hf_",i)][-train_ind,])
  model <- load_model_hdf5(paste0("./miko_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_miko[-train_ind,])
  acc_fun(predictions, data1_prepared[paste0("hf_",i)][-train_ind,])
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableE3 <- rbind(tableE3, df_temp)
}

round(tableE3[,-1], digits=2)

# Exporting table to txt
stargazer(tableE3, summary = FALSE, out = "./tables/tableE3.txt", digits = 2)


#### Appendix F ##########################################################
##### Load Data #################################################################

load("./data1_prepared.Rdata")


##### Create Training and Test Data ############################################

set.seed(1111)
sample_size <- floor(.90 * nrow(data1_prepared))
train_ind <- sample(nrow(data1_prepared), size = sample_size)

test_data <- data1_prepared[-train_ind,]
training_data <- data1_prepared[train_ind,]



# Create helper functions to calculate Accuracy, Precision, and Recall
## function to compute accuracy
acc_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(sum(diag(tab))/sum(tab))
}
# function to compute precision
prec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return((tab[2,2])/(tab[2,1]+tab[2,2]))
}
# function to compute recall
rec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(tab[2,2]/(tab[1,2]+tab[2,2]))
}



##### Table F1: Random Tree ##################################################################
# In this appendix, we replace the neural network classifier with different machine learning classifiers
# We start with Random Tree classifier

# First, we build a corpus of our training and test data
cgcorpus <- corpus(data1_prepared$Text)
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

#First, we will convert the word embeddings to a data frame, and then we will 
#match the features from each document with their corresponding embeddings.

# extracting word embeddings for words in corpus (local word embeddings)
w2v <- readr::read_delim("./vec_ed_preprocessed.txt", 
                         skip=1, delim=" ", quote="",
                         col_names=c("word", paste0("V", 1:100)))

# Stemming
w2v$word <- text_tokens(w2v$word, stemmer = "de")

# Matching
w2v <- w2v[w2v$word %in% featnames(cgdfm),]


# creating new feature matrix for embeddings
embed <- matrix(NA, nrow=ndoc(cgdfm), ncol=100)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embeddings for those words
  embed_vec <- w2v[w2v$word %in% doc_words, 2:101]
  # aggregate from word- to document-level embeddings by taking AVG
  embed[i,] <- colMeans(embed_vec, na.rm=TRUE)
  # if no words in embeddings, simply set to 0
  if (nrow(embed_vec)==0) embed[i,] <- 0
}

# After we created the sentence embeddings, we apply the trained machine learning models for each emotion
# The models for the random forrest classifier are named "rFanger.Rdata", "rFfear.Rdata", etc.

tableF1 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  load(paste0("./rF",i,".RData"))
  predictions <- predict(rfc, newdata = as.data.frame(as.matrix(embed[-train_ind,])))
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(predictions==1)
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableF1 <- rbind(tableF1, df_temp)
}

round(tableF1[,-1], digits=2)

# Exporting table to txt
stargazer(tableF1, summary = FALSE, out = "./tables/tableF1.txt", digits = 2)

##### Table F2: Lasso ###############################################################

# After we created the sentence embeddings, we apply the trained machine learning models for each emotion
# The models for the Lasso classifier are named "lasso_anger.Rdata", "lasso_fear.Rdata", etc.

tableF2 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  load(paste0("./lasso_",i,".RData"))
  predictions <- predict(lasso, embed[-train_ind,], type="class")  
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(predictions==1)
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableF2 <- rbind(tableF2, df_temp)
}

round(tableF2[,-1], digits=2)

# Exporting table to txt
stargazer(tableF2, summary = FALSE, out = "./tables/tableF2.txt", digits = 2)


##### Table F3: Naive Bayes ###############################################################

# After we created the sentence embeddings, we apply the trained machine learning models for each emotion
# The models for the random forrest classifier are named "nb_anger.Rdata", "nb_fear.Rdata", etc.

tableF3 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  load(paste0("./nb_",i,".RData"))
  predictions <- predict(classifier_nb, newdata = as.data.frame(as.matrix(embed[-train_ind,])))
  precision <- prec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  recall <- rec_fun(predictions==1, data1_prepared[paste0("hf_",i)][-train_ind,]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data1_prepared[paste0("hf_",i)][-train_ind,])
  n_pred <- sum(predictions==1)
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableF3 <- rbind(tableF3, df_temp)
}

round(tableF3[,-1], digits=2)

# Exporting table to txt
stargazer(tableF3, summary = FALSE, out = "./tables/tableF3.txt", digits = 2)



#### Appendix G ###########################################################
##### Load Data #################################################################

load("./data1_prepared.Rdata")
load("./test_data.Rdata")
load("./embed.Rdata")
load("./embed_poly.Rdata")
load("./embed_boja.Rdata")
load("./embed_miko.Rdata")




##### Table G1: ed8 ###################################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the ed8 dictionary (df_...)

tableG1 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- test_data[paste0("df_",i)]
  true <- test_data[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableG1 <- rbind(tableG1, df_temp)
}

print(tableG1)


# Exporting table to txt
stargazer(tableG1, summary = FALSE, out = "./tables/tableG1.txt")

##### Table G2: Word Embeddings #########################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by word embeddings approach (wb....)

tableG2 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- test_data[paste0("wb.",i)]
  true <- test_data[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableG2 <- rbind(tableG2, df_temp)
}

print(tableG2)


# Exporting table to txt
stargazer(tableG2, summary = FALSE, out = "./tables/tableG2.txt")

##### Table G3: ELECTRA #########################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the ELECTRA model (el_...)

tableG3 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- test_data[paste0("el_",i)]
  true <- test_data[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableG3 <- rbind(tableG3, df_temp)
}


print(tableG3)

# Exporting table to txt
stargazer(tableG3, summary = FALSE, out = "./tables/tableG3.txt")

##### Figure G1 - G8 ############################################################################

# In Figures G1 - G8 we present the ‘receiver operating characteristic’ (ROC) curves, 
# showing how sensitivity (i.e., true positive rate) and specificity 
# (i.e., true negative rate) of the predictions of the different machine learning 
# approaches vary across cutoffs for the predicted probability.
# To do this, we first load in the different classifiers for each emotion, apply them to the 
# test data and then extract the probabilities. The loop below does this automatically for us.
# We then use the ROCit package to visually display the ROC curves.
# Note: The probabilities for the ELECTRA model need to be imported first, because we 
# apply the ELECTRA model in Python. 


## Load in the probabilities from the ELECTRA model because the ELECTRA model is applied in Python
load("./prob_elek.Rdata")

############
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

j <- 1

for (i in emotion_list) {
  
  #Load models
  model <- load_model_hdf5(paste0("./keras_",i,"90"), custom_objects = NULL, compile = TRUE)
  model2 <- load_model_hdf5(paste0("./boja_keras_",i), custom_objects = NULL, compile = TRUE)
  model3 <- load_model_hdf5(paste0("./miko_keras_",i), custom_objects = NULL, compile = TRUE)
  model4 <- load_model_hdf5(paste0("./poly_keras_",i), custom_objects = NULL, compile = TRUE)
  load(paste0("./nb_",i,".RData"))
  
  #Apply neural network classifier based on locally trained word embeddings
  x_test <- embed
  y_test <- to_categorical(test_data[paste0("hf_",i)])
  model %>% predict_classes(x_test)
  predictions <- model %>% predict_classes(x_test)
  #Extract probabilities
  df <- as.data.frame(predict_proba(object = model,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj1 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  #Apply neural network classifier based on word embeddings from Bojanowski et al. 2017
  x_test <- embed_boja[-train_ind,]
  y_test <- to_categorical(test_data[paste0("hf_",i)])
  model2 %>% predict_classes(x_test)
  predictions <- model2 %>% predict_classes(x_test)
  #Extract probabilities
  df <- as.data.frame(predict_proba(object = model2,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj2 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  #Apply neural network classifier based on word embeddings from Mikolov et al. 2017
  x_test <- embed_miko[-train_ind,]
  y_test <- to_categorical(test_data[paste0("hf_",i)])
  model3 %>% predict_classes(x_test)
  predictions <- model3 %>% predict_classes(x_test)
  #Extract probabilities
  df <- as.data.frame(predict_proba(object = model3,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj3 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  #Apply neural network classifier based on Polyglot word embeddings
  x_test <- embed_poly[-train_ind,]
  y_test <- to_categorical(test_data[paste0("hf_",i)])
  model4 %>% predict_classes(x_test)
  predictions <- model4 %>% predict_classes(x_test)
  #Extract probabilities
  df <- as.data.frame(predict_proba(object = model4,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj4 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  #Apply Naive Bayes classifier based on locally trained word embeddings
  y_pred2 <- predict(classifier_nb, newdata = as.data.frame(as.matrix(embed)), "raw")
  df <- as.data.frame(y_pred2)
  #Extract probabilities
  truth <- to_categorical(test_data[paste0("hf_",i)])
  df$labels <- truth[,1]
  colnames(df)[1] <- "V1"
  colnames(df)[2] <- "V2"
  ROCit_obj5 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  #Use the probabilities extracted from the ELECTRA Model
  df_elek <- prob_elek[,c(paste0(i),paste0("hf_",i))]
  ROCit_obj6 <- rocit(score=df_elek[,1],class=df_elek[,2], method = "emp")
  
  #Plot the individual graphs
  pdf(paste0("./figures/FigureG",j,".pdf"))
  plot(ROCit_obj1,
       legend = FALSE, YIndex = FALSE)
  lines(ROCit_obj2$TPR~ROCit_obj2$FPR, 
        col = 2, lwd = 2)
  lines(ROCit_obj3$TPR~ROCit_obj3$FPR, 
        col = 4, lwd = 2)
  lines(ROCit_obj4$TPR~ROCit_obj4$FPR, 
        col = 6, lwd = 2)
  lines(ROCit_obj5$TPR~ROCit_obj5$FPR, 
        col = 8, lwd = 2)
  lines(ROCit_obj6$TPR~ROCit_obj6$FPR, 
        col = "green", lwd = 2)
  legend("bottomright", col = c(1,2,4,6,8, "green"),
         c("Word embeddings (local)", "Bojanowski et al. 2017",
           "Mikolov et al. 2017", "Polyglot", "Naive Bayes", "ELECTRA"), lwd = 2)
  title(main = i)
  my_plot <- recordPlot() 
  #Assign a name
  assign(paste0("roc_",i), my_plot)
  dev.off() 
  j <- j + 1
  
}

# Now, you can look at the individual plots
roc_anger        #FigureG1
roc_fear         #FigureG2
roc_disgust      #FigureG3
roc_sadness      #FigureG4
roc_joy          #FigureG5
roc_enthusiasm   #FigureG6
roc_pride        #FigureG7
roc_hope         #FigureG8


#### Appendix H #########################################################
##### Load Data #################################################################

load("./test_data.Rdata")



##### New Validation Split #####################################################
# Split test data by source (Facebook and Parliamentary Speeches)
vdata_fb <- test_data[test_data$type=="fb_sent",]
vdata_ps <- test_data[test_data$type=="ps_sent",]


##### Table H1 ###############################################################
# In the following loops, we take the binary variables produced by the different tools (ed8, word embeddings, ELECTRA)
# as predictions and the human judgement as "true" data
# then we calculate recall, precision and F1 scores for both text sources
# The loops below do this automatically, the results can be printed below

# ed8 dictionary
df_ed8 <- NULL
df_wb <- NULL
df_elek <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict_facebook <- vdata_fb[paste0("df_",i)]
  true_facebook <- vdata_fb[paste0("hf_",i)]
  
  retrieved_facebook <- sum(predict_facebook)
  prec_facebook <- sum(predict_facebook & true_facebook) / retrieved_facebook
  rec_facebook <- sum(predict_facebook & true_facebook) / sum(true_facebook)
  fscore_facebook <- 2 * prec_facebook * rec_facebook / (prec_facebook + rec_facebook)
  
  emotion <- i
  tool <- "ed8"
  precision_facebook <- prec_facebook
  recall_facebook <- rec_facebook
  Fmeasure_facebook <- fscore_facebook
  actual_facebook <- sum(true_facebook)
  predicted_facebook <- sum(predict_facebook)
  
  predict_parliament <- vdata_ps[paste0("df_",i)]
  true_parliament <- vdata_ps[paste0("hf_",i)]
  
  retrieved_parliament <- sum(predict_parliament)
  prec_parliament <- sum(predict_parliament & true_parliament) / retrieved_parliament
  rec_parliament <- sum(predict_parliament & true_parliament) / sum(true_parliament)
  fscore_parliament <- 2 * prec_parliament * rec_parliament / (prec_parliament + rec_parliament)
  
  emotion <- i
  tool <- "ed8"
  precision_parliament <- prec_parliament
  recall_parliament <- rec_parliament
  Fmeasure_parliament <- fscore_parliament
  actual_parliament <- sum(true_parliament)
  predicted_parliament <- sum(predict_parliament)
  
  df_temp <- data.frame(tool, emotion, precision_facebook, 
                        recall_facebook, Fmeasure_facebook, precision_parliament, 
                        recall_parliament, Fmeasure_parliament)
  df_ed8 <- rbind(df_ed8, df_temp)
}


# Word Embeddings
for (i in emotion_list) {
  predict_facebook <- vdata_fb[paste0("wb.",i)]
  true_facebook <- vdata_fb[paste0("hf_",i)]
  
  retrieved_facebook <- sum(predict_facebook)
  prec_facebook <- sum(predict_facebook & true_facebook) / retrieved_facebook
  rec_facebook <- sum(predict_facebook & true_facebook) / sum(true_facebook)
  fscore_facebook <- 2 * prec_facebook * rec_facebook / (prec_facebook + rec_facebook)
  
  emotion <- i
  tool <- "wb"
  precision_facebook <- prec_facebook
  recall_facebook <- rec_facebook
  Fmeasure_facebook <- fscore_facebook
  actual_facebook <- sum(true_facebook)
  predicted_facebook <- sum(predict_facebook)
  
  predict_parliament <- vdata_ps[paste0("wb.",i)]
  true_parliament <- vdata_ps[paste0("hf_",i)]
  
  retrieved_parliament <- sum(predict_parliament)
  prec_parliament <- sum(predict_parliament & true_parliament) / retrieved_parliament
  rec_parliament <- sum(predict_parliament & true_parliament) / sum(true_parliament)
  fscore_parliament <- 2 * prec_parliament * rec_parliament / (prec_parliament + rec_parliament)
  
  emotion <- i
  tool <- "wb"
  precision_parliament <- prec_parliament
  recall_parliament <- rec_parliament
  Fmeasure_parliament <- fscore_parliament
  actual_parliament <- sum(true_parliament)
  predicted_parliament <- sum(predict_parliament)
  
  df_temp <- data.frame(tool, emotion, precision_facebook, 
                        recall_facebook, Fmeasure_facebook, precision_parliament, 
                        recall_parliament, Fmeasure_parliament)
  df_wb <- rbind(df_wb, df_temp)
}


# ELECTRA
for (i in emotion_list) {
  predict_facebook <- vdata_fb[paste0("el_",i)]
  true_facebook <- vdata_fb[paste0("hf_",i)]
  
  retrieved_facebook <- sum(predict_facebook)
  prec_facebook <- sum(predict_facebook & true_facebook) / retrieved_facebook
  rec_facebook <- sum(predict_facebook & true_facebook) / sum(true_facebook)
  fscore_facebook <- 2 * prec_facebook * rec_facebook / (prec_facebook + rec_facebook)
  
  emotion <- i
  tool <- "elek"
  precision_facebook <- prec_facebook
  recall_facebook <- rec_facebook
  Fmeasure_facebook <- fscore_facebook
  actual_facebook <- sum(true_facebook)
  predicted_facebook <- sum(predict_facebook)
  
  predict_parliament <- vdata_ps[paste0("el_",i)]
  true_parliament <- vdata_ps[paste0("hf_",i)]
  
  retrieved_parliament <- sum(predict_parliament)
  prec_parliament <- sum(predict_parliament & true_parliament) / retrieved_parliament
  rec_parliament <- sum(predict_parliament & true_parliament) / sum(true_parliament)
  fscore_parliament <- 2 * prec_parliament * rec_parliament / (prec_parliament + rec_parliament)
  
  emotion <- i
  tool <- "Electra"
  precision_parliament <- prec_parliament
  recall_parliament <- rec_parliament
  Fmeasure_parliament <- fscore_parliament
  actual_parliament <- sum(true_parliament)
  predicted_parliament <- sum(predict_parliament)
  
  df_temp <- data.frame(tool, emotion, precision_facebook, 
                        recall_facebook, Fmeasure_facebook, precision_parliament, 
                        recall_parliament, Fmeasure_parliament)
  df_elek <- rbind(df_elek, df_temp)
}

tableH1 <- rbind(df_ed8, df_wb, df_elek)

print(tableH1)   

# Exporting table to txt
stargazer(tableH1, summary = FALSE, out = "./tables/tableH1.txt", digits = 2)


##### Table H2 ###############################################################################
# In the following loops, we take the binary variables produced by the different tools (ed8, word embeddings, ELECTRA)
# as predictions and the human judgement as "true" data
# then we calculate the number of predicted and "true" occurences
# The loops below do this automatically, the results can be printed below

df_facebook <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  emotion <- i
  
  predict_ed8 <- vdata_fb[paste0("df_",i)]
  true_facebook <- vdata_fb[paste0("hf_",i)]
  
  actual_facebook <- sum(true_facebook)
  predict_ed8 <- sum(predict_ed8)
  
  predict_wb <- vdata_fb[paste0("wb.",i)]
  predict_wb <- sum(predict_wb)
  
  predict_electra <- vdata_fb[paste0("el_",i)]
  predict_electra <- sum(predict_electra)
  
  df_temp <- data.frame(actual_facebook, predict_ed8, predict_wb, predict_electra)
  df_facebook <- rbind(df_facebook, df_temp)
}

print(df_facebook)

df_parliament <- NULL
for (i in emotion_list) {
  emotion <- i
  
  predict_ed8 <- vdata_ps[paste0("df_",i)]
  true_parliament <- vdata_ps[paste0("hf_",i)]
  
  actual_parliament <- sum(true_parliament)
  predict_ed8 <- sum(predict_ed8)
  
  predict_wb <- vdata_ps[paste0("wb.",i)]
  predict_wb <- sum(predict_wb)
  
  predict_electra <- vdata_ps[paste0("el_",i)]
  predict_electra <- sum(predict_electra)
  
  df_temp <- data.frame(actual_parliament, predict_ed8, predict_wb, predict_electra)
  df_parliament <- rbind(df_parliament, df_temp)
}

print(df_parliament)

tableH2 <- cbind(df_facebook, df_parliament)

print(tableH2)
# Exporting table to txt
stargazer(tableH2, summary = FALSE, out = "./tables/tableH2.txt")


#### Appendix I ##########################################################
##### Load Data #################################################################
load("./data1_prepared.Rdata")
load("./test_data.Rdata")


##### Table I1: NRC dictionary ###################################################################################
# To create the confusion matrix in Table I1 we simply count the number of emotional occurrences
# as classified by the human annotators (hf_...) and by the NRC dictionary (nf_...)

tableI1 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy")

for (i in emotion_list) {
  predict <- test_data[paste0("nf_",i)]
  true <- test_data[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableI1 <- rbind(tableI1, df_temp)
}

print(tableI1)

# Exporting table to txt
stargazer(tableI1, summary = FALSE, out = "./tables/tableI1.txt")




##### Table I2: LIWC dictionary ###################################################################################
# To create the confusion matrix in Table I2 we simply count the number of emotional occurrences
# as classified by the human annotators (hf_...) and by the LIWC dictionary (lf_...)

tableI2 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "sadness")

for (i in emotion_list) {
  predict <- test_data[paste0("lf_",i)]
  true <- test_data[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableI2 <- rbind(tableI2, df_temp)
}

print(tableI2)

# Exporting table to txt
stargazer(tableI2, summary = FALSE, out = "./tables/tableI2.txt")


##### Table I3 ###################################################################################
# In Table I3 we calculate precision, recall, and F1 scores based on the training + test data (9898 sentences)
# Table I3 presents the performance of the ed8 dictionary (df_...)

tableI3 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- data1_prepared[paste0("df_",i)]
  true <- data1_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "ed8"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableI3 <- rbind(tableI3, df_temp)
}

print(tableI3)

# Exporting table to txt
stargazer(tableI3, summary = FALSE, out = "./tables/tableI3.txt", digits = 2)


##### Table I4 #########################################################################
# In Table I4 we calculate precision, recall, and F1 scores based on the training + test data (9898 sentences)
# Table I4 presents the performance of the LIWC dictionary (lf_...)

tableI4 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "sadness")

for (i in emotion_list) {
  predict <- data1_prepared[paste0("lf_",i)]
  true <- data1_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "LIWC"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableI4 <- rbind(tableI4, df_temp)
}

print(tableI4)

# Exporting table to txt
stargazer(tableI4, summary = FALSE, out = "./tables/tableI4.txt", digits = 2)


##### Table I5 #########################################################################
# In Table I5 we calculate precision, recall, and F1 scores based on the training + test data (9898 sentences)
# Table I5 presents the performance of the NRC dictionary (nf_...)

tableI5 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy")

for (i in emotion_list) {
  predict <- data1_prepared[paste0("nf_",i)]
  true <- data1_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "NRC"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableI5 <- rbind(tableI5, df_temp)
}

print(tableI5)

# Exporting table to txt
stargazer(tableI5, summary = FALSE, out = "./tables/tableI5.txt", digits = 2)


##### Figure I1 #########################################################################
# To create Figure I1, we first need to create categorical variables
# We assign "clearly emotional" to sentences which have been coded as 'emotional' by 5 or 4 human coders
# We assign "emotional" to sentences which have been coded as 'emotional' by 3 or 2 human coders
# We assign "not/slightly emotion" to sentences which have been coded as 'emotional' by 0 or 1 human coders


## Categorical
data1_prepared$h_anger4 <- NULL
data1_prepared$h_anger4[data1_prepared$h_anger==5 | data1_prepared$h_anger == 4] <- "clearly emotional"
data1_prepared$h_anger4[data1_prepared$h_anger==3 | data1_prepared$h_anger == 2] <- "emotional"
data1_prepared$h_anger4[data1_prepared$h_anger == 0 | data1_prepared$h_anger == 1] <- "not/slightly"

data1_prepared$h_fear4 <- NULL
data1_prepared$h_fear4[data1_prepared$h_fear==5 | data1_prepared$h_fear == 4] <- "clearly emotional"
data1_prepared$h_fear4[data1_prepared$h_fear==3 | data1_prepared$h_fear == 2] <- "emotional"
data1_prepared$h_fear4[data1_prepared$h_fear == 0 | data1_prepared$h_fear == 1] <- "not/slightly"

data1_prepared$h_disgust4 <- NULL
data1_prepared$h_disgust4[data1_prepared$h_disgust==5 | data1_prepared$h_disgust == 4] <- "clearly emotional"
data1_prepared$h_disgust4[data1_prepared$h_disgust==3 | data1_prepared$h_disgust == 2] <- "emotional"
data1_prepared$h_disgust4[data1_prepared$h_disgust == 0 | data1_prepared$h_disgust == 1] <- "not/slightly"

data1_prepared$h_sadness4 <- NULL
data1_prepared$h_sadness4[data1_prepared$h_sadness==5 | data1_prepared$h_sadness == 4] <- "clearly emotional"
data1_prepared$h_sadness4[data1_prepared$h_sadness==3 | data1_prepared$h_sadness == 2] <- "emotional"
data1_prepared$h_sadness4[data1_prepared$h_sadness == 0 | data1_prepared$h_sadness == 1] <- "not/slightly"

data1_prepared$h_joy4 <- NULL
data1_prepared$h_joy4[data1_prepared$h_joy==5 | data1_prepared$h_joy == 4] <- "clearly emotional"
data1_prepared$h_joy4[data1_prepared$h_joy==3 | data1_prepared$h_joy == 2] <- "emotional"
data1_prepared$h_joy4[data1_prepared$h_joy == 0 | data1_prepared$h_joy == 1] <- "not/slightly"

data1_prepared$h_enthusiasm4 <- NULL
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm==5 | data1_prepared$h_enthusiasm == 4] <- "clearly emotional"
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm==3 | data1_prepared$h_enthusiasm == 2] <- "emotional"
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm == 0 | data1_prepared$h_enthusiasm == 1] <- "not/slightly"

data1_prepared$h_pride4 <- NULL
data1_prepared$h_pride4[data1_prepared$h_pride==5 | data1_prepared$h_pride == 4] <- "clearly emotional"
data1_prepared$h_pride4[data1_prepared$h_pride==3 | data1_prepared$h_pride == 2] <- "emotional"
data1_prepared$h_pride4[data1_prepared$h_pride == 0 | data1_prepared$h_pride == 1] <- "not/slightly"

data1_prepared$h_hope4 <- NULL
data1_prepared$h_hope4[data1_prepared$h_hope==5 | data1_prepared$h_hope == 4] <- "clearly emotional"
data1_prepared$h_hope4[data1_prepared$h_hope==3 | data1_prepared$h_hope == 2] <- "emotional"
data1_prepared$h_hope4[data1_prepared$h_hope == 0 | data1_prepared$h_hope == 1] <- "not/slightly"


#Then we order the variable levels so the graphs go from the lowest to the highest value
data1_prepared$h_anger4 <- ordered(data1_prepared$h_anger4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_fear4 <- ordered(data1_prepared$h_fear4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_disgust4 <- ordered(data1_prepared$h_disgust4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_sadness4 <- ordered(data1_prepared$h_sadness4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_joy4 <- ordered(data1_prepared$h_joy4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_enthusiasm4 <- ordered(data1_prepared$h_enthusiasm4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_pride4 <- ordered(data1_prepared$h_pride4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_hope4 <- ordered(data1_prepared$h_hope4,levels = c("not/slightly", "emotional", "clearly emotional"))



### Anger

pd <- position_dodge(0.1) # move them .1 to the left and right

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_nrc <- select(data1_prepared, nrc.anger.norm, h_anger4)
data1_prepared_nrc$dic <- "NRC"
colnames(data1_prepared_nrc)[1] <- "anger.norm"
data1_prepared_ed8 <- select(data1_prepared, anger.norm, h_anger4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared_liwc <- select(data1_prepared, liwc.anger.norm, h_anger4)
data1_prepared_liwc$dic <- "LIWC"
colnames(data1_prepared_liwc)[1] <- "anger.norm"
data1_prepared2 <- rbind(data1_prepared_nrc, data1_prepared_ed8, data1_prepared_liwc)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="anger.norm", groupvars=c("h_anger4", "dic"), na.rm=TRUE)

#Display relationship graphically
anger <- ggplot(df2, aes(x=h_anger4, y=anger.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=anger.norm-ci, ymax=anger.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("ANGER \n(Normalized emotional scores)") +
  labs(color='Dictionary') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### fear

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_nrc <- select(data1_prepared, nrc.fear.norm, h_fear4)
data1_prepared_nrc$dic <- "NRC"
colnames(data1_prepared_nrc)[1] <- "fear.norm"
data1_prepared_ed8 <- select(data1_prepared, fear.norm, h_fear4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared_liwc <- select(data1_prepared, liwc.fear.norm, h_fear4)
data1_prepared_liwc$dic <- "LIWC"
colnames(data1_prepared_liwc)[1] <- "fear.norm"
data1_prepared2 <- rbind(data1_prepared_nrc, data1_prepared_ed8, data1_prepared_liwc)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="fear.norm", groupvars=c("h_fear4", "dic"), na.rm=TRUE)

#Display relationship graphically
fear <- ggplot(df2, aes(x=h_fear4, y=fear.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=fear.norm-ci, ymax=fear.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("fear \n(Normalized emotional scores)") +
  labs(color='Dictionary') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### disgust

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_nrc <- select(data1_prepared, nrc.disgust.norm, h_disgust4)
data1_prepared_nrc$dic <- "NRC"
colnames(data1_prepared_nrc)[1] <- "disgust.norm"
data1_prepared_ed8 <- select(data1_prepared, disgust.norm, h_disgust4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared2 <- rbind(data1_prepared_nrc, data1_prepared_ed8)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="disgust.norm", groupvars=c("h_disgust4", "dic"), na.rm=TRUE)

#Display relationship graphically
disgust <- ggplot(df2, aes(x=h_disgust4, y=disgust.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=disgust.norm-ci, ymax=disgust.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("disgust \n(Normalized emotional scores)") +
  labs(color='Dictionary') +
  scale_color_manual(values=c("#F8766D", "#7CAE00"))

### sadness

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_nrc <- select(data1_prepared, nrc.sadness.norm, h_sadness4)
data1_prepared_nrc$dic <- "NRC"
colnames(data1_prepared_nrc)[1] <- "sadness.norm"
data1_prepared_ed8 <- select(data1_prepared, sadness.norm, h_sadness4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared_liwc <- select(data1_prepared, liwc.sad.norm, h_sadness4)
data1_prepared_liwc$dic <- "LIWC"
colnames(data1_prepared_liwc)[1] <- "sadness.norm"
data1_prepared2 <- rbind(data1_prepared_nrc, data1_prepared_ed8, data1_prepared_liwc)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="sadness.norm", groupvars=c("h_sadness4", "dic"), na.rm=TRUE)

#Display relationship graphically
sadness <- ggplot(df2, aes(x=h_sadness4, y=sadness.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=sadness.norm-ci, ymax=sadness.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("sadness \n(Normalized emotional scores)") +
  labs(color='Dictionary') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))


figureI1 <- ggarrange(anger, fear, disgust, sadness,
                      ncol = 2, nrow = 2)

figureI1 

ggsave("./figures/figureI1.pdf")



##### Figure I2 ##############################################################

#### Joy

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_nrc <- select(data1_prepared, nrc.joy.norm, h_joy4)
data1_prepared_nrc$dic <- "NRC"
colnames(data1_prepared_nrc)[1] <- "joy.norm"
data1_prepared_ed8 <- select(data1_prepared, joy.norm, h_joy4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared2 <- rbind(data1_prepared_nrc, data1_prepared_ed8)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="joy.norm", groupvars=c("h_joy4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

#Display relationship graphically
joy <- ggplot(df2, aes(x=h_joy4, y=joy.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=joy.norm-ci, ymax=joy.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("JOY \n(Normalized emotional scores)") +
  labs(color='Dictionary')  +
  scale_color_manual(values=c("#F8766D", "#7CAE00"))

#### Enthusiasm

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_ed8 <- select(data1_prepared, enthusiasm.norm, h_enthusiasm4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared2 <- rbind(data1_prepared_ed8)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="enthusiasm.norm", groupvars=c("h_enthusiasm4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

#Display relationship graphically
enthusiasm <- ggplot(df2, aes(x=h_enthusiasm4, y=enthusiasm.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=enthusiasm.norm-ci, ymax=enthusiasm.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("ENTHUSIASM \n(Normalized emotional scores)") +
  labs(color='Dictionary')  +
  scale_color_manual(values=c("#F8766D", "#7CAE00"))

### Pride

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_ed8 <- select(data1_prepared, pride.norm, h_pride4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared2 <- rbind(data1_prepared_ed8)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="pride.norm", groupvars=c("h_pride4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

#Display relationship graphically
pride <- ggplot(df2, aes(x=h_pride4, y=pride.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=pride.norm-ci, ymax=pride.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("PRIDE \n(Normalized emotional scores)") +
  labs(color='Dictionary')  +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Hope

#Create one data frame for all three dictionaries with normalized emotional score by category
data1_prepared_ed8 <- select(data1_prepared, hope.norm, h_hope4)
data1_prepared_ed8$dic <- "ed8"
data1_prepared2 <- rbind(data1_prepared_ed8)

#calculate standard error, standard deviation, and confidence intervals
df2 <- summarySE(data1_prepared2, measurevar="hope.norm", groupvars=c("h_hope4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

#Display relationship graphically
hope <- ggplot(df2, aes(x=h_hope4, y=hope.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=hope.norm-ci, ymax=hope.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("HOPE \n(Normalized emotional scores)") +
  labs(color='Dictionary')  +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

figureI2 <- ggarrange(joy, enthusiasm, pride, hope,
                     ncol = 2, nrow = 2)

figureI2

ggsave("./figures/figureI2.pdf")


#### Appendix J #########################################################
##### Load Data #################################################################
load("./data2_prepared.RData")


## function to compute accuracy
acc_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(sum(diag(tab))/sum(tab))
}
# function to compute precision
prec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return((tab[2,2])/(tab[2,1]+tab[2,2]))
}
# function to compute recall
rec_fun <- function(ypred, y){
  tab <- table(ypred, y)
  return(tab[2,2]/(tab[1,2]+tab[2,2]))
}


##### Table J1 ##################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# We start with the using the binary variable based on the ed8 dictionary (df_...) and 
# compare it to human judgement (hf_...)

# ed8 Dictionary
tableJ1 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("df_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "ed8"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableJ1 <- rbind(tableJ1, df_temp)
}

print(tableJ1)

# Exporting table to txt
stargazer(tableJ1, summary = FALSE, out = "./tables/tableJ1.txt", digits = 2)

##### Table J2 ##################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the word embeddings approach (wb....) and 
# compare it to human judgement (hf_...)

# Word Embeddings
# First, we create a corpus and pre-process this corpus
cgcorpus <- corpus(data2_prepared$Text)
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

# Then, we will convert the word embeddings to a data frame,
# and match the features from each document with their corresponding embeddings.

w2v <- readr::read_delim("./vec_ed_preprocessed.txt", 
                         skip=1, delim=" ", quote="",
                         col_names=c("word", paste0("V", 1:100)))

# Stem the word
w2v$word <- text_tokens(w2v$word, stemmer = "de")

# extracting word embeddings for words in corpus
w2v <- w2v[w2v$word %in% featnames(cgdfm),]


# creating new feature matrix for embeddings
embed2 <- matrix(NA, nrow=ndoc(cgdfm), ncol=100)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embed2dings for those words
  embed2_vec <- w2v[w2v$word %in% doc_words, 2:101]
  # aggregate from word- to document-level embed2dings by taking AVG
  embed2[i,] <- colMeans(embed2_vec, na.rm=TRUE)
  # if no words in embed2dings, simply set to 0
  if (nrow(embed2_vec)==0) embed2[i,] <- 0
}

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion

tableJ2 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  model <- load_model_hdf5(paste0("./keras_",i,"90"), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed2)
  precision <- prec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  recall <- rec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data2_prepared[paste0("hf_",i)])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableJ2 <- rbind(tableJ2, df_temp)
}

round(tableJ2[,-1], digits=2)

# Exporting table to txt
stargazer(tableJ2, summary = FALSE, out = "./tables/tableJ2.txt", digits = 2)


##### Table J3 ##################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the ELECTRA approach (el_...) and 
# compare it to human judgement (hf_...)

#ELECTRA
tableJ3 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("el_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "ELECTRA"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableJ3 <- rbind(tableJ3, df_temp)
}

print(tableJ3)

# Exporting table to txt
stargazer(tableJ3, summary = FALSE, out = "./tables/tableJ3.txt", digits = 2)

##### Table J4 ##################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the LIWC dictionary (lf_...) and 
# compare it to human judgement (hf_...)

# LIWC Dictionary
tableJ4 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "sadness")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("lf_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "LIWC"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableJ4 <- rbind(tableJ4, df_temp)
}

print(tableJ4)

# Exporting table to txt
stargazer(tableJ4, summary = FALSE, out = "./tables/tableJ4.txt", digits = 2)


##### Table J5 ##################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the NRC dictionary (nf_...) and 
# compare it to human judgement (hf_...)

# NRC Dictionary
tableJ5 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("nf_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  
  retrieved <- sum(predict)
  prec <- sum(predict & true) / retrieved
  rec <- sum(predict & true) / sum(true)
  fscore <- 2 * prec * rec / (prec + rec)
  
  emotion <- i
  tool <- "NRC"
  precision <- prec
  recall <- rec
  Fmeasure <- fscore
  actual <- sum(true)
  predicted <- sum(predict)
  
  df_temp <- data.frame(tool, emotion, actual, predicted, precision, recall, Fmeasure)
  tableJ5 <- rbind(tableJ5, df_temp)
}

print(tableJ5)

# Exporting table to txt
stargazer(tableJ5, summary = FALSE, out = "./tables/tableJ5.txt", digits = 2)

##### Table J6 ###############################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the Bojanowski et al. (2017) embeddings and 
# compare it to human judgement (hf_...)


#### Bojanowski et al 2017

# First, we create a corpus and pre-process this corpus
cgcorpus <- corpus(data2_prepared$Text)
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)

# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

# Then, we will convert the word embeddings to a data frame,
# and match the features from each document with their corresponding embeddings.

# To do so, we first load the embeddings by Bojakowski et al. 2017
# Note, this step takes a very long time can can be skipped, simply load the file "embed_boja2.Rdata" below
lines <- readLines("./boja_wiki.de.vec")

df1 <- read.table(text=lines, sep="", quote="", fill = TRUE)

colnames(df1)[1] <- "word"

# Stemming
df1$word <- text_tokens(df1$word, stemmer = "de")

# Matching
w2v <- df1[df1$word %in% featnames(cgdfm),]

# creating new feature matrix for embed_v2dings
embed_v2 <- matrix(NA, nrow=ndoc(cgdfm), ncol=300)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embed_v2dings for those words
  embed_v2_vec <- w2v[w2v$word %in% doc_words, 2:301]
  # aggregate from word- to document-level embed_v2dings by taking AVG
  embed_v2[i,] <- colMeans(embed_v2_vec, na.rm=TRUE)
  # if no words in embed_v2dings, simply set to 0
  if (nrow(embed_v2_vec)==0) embed_v2[i,] <- 0
}

embed_boja2 <- embed_v2

load("./embed_boja2.Rdata")

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion

tableJ6 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_boja2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  model <- load_model_hdf5(paste0("./boja_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_boja2)
  precision <- prec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  recall <- rec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data2_prepared[paste0("hf_",i)])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableJ6 <- rbind(tableJ6, df_temp)
}

round(tableJ6[,-1], digits=2)

# Exporting table to txt
stargazer(tableJ6, summary = FALSE, out = "./tables/tableJ6.txt", digits = 2)


##### Table J7 ###############################################################################
# We calculate precision, recall, and F1 scores for the randomly sampled dataset.
# Here, we use the binary variable based on the Mikolov et al. (2017) embeddings and 
# compare it to human judgement (hf_...)

# First, we create a corpus and pre-process this corpus
cgcorpus <- corpus(data2_prepared$Text)
cgdfm <- dfm(cgcorpus, remove=stopwords("german"), verbose=TRUE, tolower = TRUE)
# Stemming
cgdfm <- dfm_wordstem(cgdfm, language = "german")

#Then, we will convert the word embeddings to a data frame, and then we will 
#match the features from each document with their corresponding embeddings.

# To do so, we first load the embeddings by Mikolov et al. 2017
# Note, this step takes a very long time can can be skipped, simply load the file "embed_miko2.Rdata" below
lines <- readLines("./miko_cc.de.300.vec")
df1 <- read.table(text=lines, sep="", quote="", fill = TRUE, skip = 1)

colnames(df1)[1] <- "word"

# Stemming
df1$word <- text_tokens(df1$word, stemmer = "de")

# Matching
w2v <- df1[df1$word %in% featnames(cgdfm),]

# creating new feature matrix for embed_v3dings
embed_v3 <- matrix(NA, nrow=ndoc(cgdfm), ncol=300)
for (i in 1:ndoc(cgdfm)){
  if (i %% 100 == 0) message(i, '/', ndoc(cgdfm))
  # extract word counts
  vec <- as.numeric(cgdfm[i,])
  # keep words with counts of 1 or more
  doc_words <- featnames(cgdfm)[vec>0]
  # extract embed_v3dings for those words
  embed_v3_vec <- w2v[w2v$word %in% doc_words, 2:301]
  # aggregate from word- to document-level embed_v3dings by taking AVG
  embed_v3[i,] <- colMeans(embed_v3_vec, na.rm=TRUE)
  # if no words in embed_v3dings, simply set to 0
  if (nrow(embed_v3_vec)==0) embed_v3[i,] <- 0
}

embed_miko2 <- embed_v3

load("./embed_miko2.Rdata")

#After we created the sentence embeddings, we apply the trained machine learning models for each emotion
#Note: the code for training of the models can be found in the file ...

tableJ7 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_miko2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  model <- load_model_hdf5(paste0("./miko_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_miko2)
  precision <- prec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  recall <- rec_fun(predictions==1, data2_prepared[paste0("hf_",i)]==1)
  Fmeasure <- 2 * precision * recall / (precision + recall)
  n_actual <- sum(data2_prepared[paste0("hf_",i)])
  n_pred <- sum(as.numeric(predictions))
  emotion <- i
  df_temp <- data.frame(emotion, n_actual, n_pred, precision, recall, Fmeasure)
  tableJ7 <- rbind(tableJ7, df_temp)
}

round(tableJ7[,-1], digits=2)

# Exporting table to txt
stargazer(tableJ7, summary = FALSE, out = "./tables/tableJ7.txt", digits = 2)

##### Table J8 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the ed8 dictionary (df_...)

tableJ8 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("df_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ8 <- rbind(tableJ8, df_temp)
}


print(tableJ8)

# Exporting table to txt
stargazer(tableJ8, summary = FALSE, out = "./tables/tableJ8.txt")


##### Table J9 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the word embedding approach (wb....)


tableJ9 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed2
  y_test <- data2_prepared[paste0("hf_",i)]
  model <- load_model_hdf5(paste0("./keras_",i,"90"), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed2)
  predict <- as.numeric(predictions)
  true <- as.numeric(y_test[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ9 <- rbind(tableJ9, df_temp)
}

print(tableJ9)

# Exporting table to txt
stargazer(tableJ9, summary = FALSE, out = "./tables/tableJ9.txt")



##### Table J10 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the ELECTRA approach (el_...)

tableJ10 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("el_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ10 <- rbind(tableJ10, df_temp)
}


print(tableJ10)

# Exporting table to txt
stargazer(tableJ10, summary = FALSE, out = "./tables/tableJ10.txt")


##### Table J11 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the NRC dictionary (nf_...)

tableJ11 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "disgust", "sadness", "joy")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("nf_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ11 <- rbind(tableJ11, df_temp)
}


print(tableJ11)

# Exporting table to txt
stargazer(tableJ11, summary = FALSE, out = "./tables/tableJ11.txt")

##### Table J12 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the LIWC dictionary (lf_...)

tableJ12 <- NULL
df_temp <- NULL
emotion_list <- c("anger", "fear", "sadness")

for (i in emotion_list) {
  predict <- data2_prepared[paste0("lf_",i)]
  true <- data2_prepared[paste0("hf_",i)]
  predict <- as.numeric(predict[,1])
  true <- as.numeric(true[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ12 <- rbind(tableJ12, df_temp)
}


print(tableJ12)

# Exporting table to txt
stargazer(tableJ12, summary = FALSE, out = "./tables/tableJ12.txt")

##### Table J13 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the Bojanowski et al. embeddings 

tableJ13 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_boja2
  y_test <- data2_prepared[paste0("hf_",i)]
  model <- load_model_hdf5(paste0("./boja_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_boja2)
  predict <- as.numeric(predictions)
  true <- as.numeric(y_test[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ13 <- rbind(tableJ13, df_temp)
}

print(tableJ13)

# Exporting table to txt
stargazer(tableJ13, summary = FALSE, out = "./tables/tableJ13.txt")

##### Table J14 ###############################################################################
# To create the confusion matrices we simply count the number of emotional occurences
# as classified by the human annotators (hf_...) and by the Mikolow et al. embeddings 

tableJ14 <- NULL
df_temp <- NULL

emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

for (i in emotion_list) {
  x_test <- embed_miko2
  y_test <- data2_prepared[paste0("hf_",i)]
  model <- load_model_hdf5(paste0("./miko_keras_",i), custom_objects = NULL, compile = TRUE)
  predictions <- model %>% predict_classes(embed_miko2)
  predict <- as.numeric(predictions)
  true <- as.numeric(y_test[,1])
  
  conf.mat <- data.frame(table(predict, true))
  emotion <- i
  df_temp <- data.frame(conf.mat, emotion)
  tableJ14 <- rbind(tableJ14, df_temp)
}

print(tableJ14)

# Exporting table to txt
stargazer(tableJ14, summary = FALSE, out = "./tables/tableJ14.txt")


##### Figure J1 - J8 #####################################################################
# In Figures J1 - J8 we present the ROC curves for the randomly sampled data set. 
# To do this, we first load in the different classifiers for each emotion, apply them to the 
# randomly sampled data set and then extract the probabilities. 
# The loop below does this automatically for us.
# We then use the ROCit package to visually display the ROC curves.
# Note: The probabilities for the ELECTRA model need to be imported first, because we 
# apply the ELECTRA model in Python. 


load("./prob_elek2.Rdata")

# We also have to load the embeddings of the randomly sampled dataset so that we can
# apply the different classifier models
load("./embed2.Rdata")


emotion_list <- c("anger", "fear", "disgust", "sadness", "joy", "enthusiasm", "pride", "hope")

j <- 1

for (i in emotion_list) {
  model <- load_model_hdf5(paste0("./keras_",i,"90"), custom_objects = NULL, compile = TRUE)
  model2 <- load_model_hdf5(paste0("./boja_keras_",i), custom_objects = NULL, compile = TRUE)
  model3 <- load_model_hdf5(paste0("./miko_keras_",i), custom_objects = NULL, compile = TRUE)
  
  x_test <- embed2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  model %>% predict_classes(x_test)
  predictions <- model %>% predict_classes(x_test)
  df <- as.data.frame(predict_proba(object = model,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj1 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  
  df_elek <- prob_elek2[,c(paste0(i),paste0("hf_",i))]
  ROCit_obj2 <- rocit(score=df_elek[,1],class=df_elek[,2], method = "emp")
  
  x_test <- embed_boja2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  predictions <- model2 %>% predict_classes(x_test)
  df <- as.data.frame(predict_proba(object = model2,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj3 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  
  x_test <- embed_miko2
  y_test <- to_categorical(data2_prepared[paste0("hf_",i)])
  predictions <- model3 %>% predict_classes(x_test)
  df <- as.data.frame(predict_proba(object = model3,
                                    x = x_test))
  truth <- as.data.frame(y_test)
  df$labels <- truth[,1]
  df <- df[,c(1,3)]
  ROCit_obj4 <- rocit(score=df$V1,class=df$labels, method = "emp")
  
  pdf(paste0("./figures/FigureJ",j,".pdf"))
  
  plot(ROCit_obj1,
       legend = FALSE, YIndex = FALSE)
  
  lines(ROCit_obj2$TPR~ROCit_obj2$FPR, 
        col = "green", lwd = 2)
  
  lines(ROCit_obj3$TPR~ROCit_obj3$FPR, 
        col = 2, lwd = 2)
  
  lines(ROCit_obj4$TPR~ROCit_obj4$FPR, 
        col = 4, lwd = 2)
  legend("bottomright", col = c(1,"green", 2, 4),
         c("Word embeddings (local)", "ELECTRA", "Bojanowski et al. 2017", 
           "Mikolov et al. 2017"), lwd = 2)
  title(main = i)
  my_plot <- recordPlot() 
  assign(paste0("roc_",i), my_plot)
  dev.off()
  j <- j + 1
}

# Now, you can look at the individual plots
roc_anger        #FigureJ1
roc_fear         #FigureJ2
roc_disgust      #FigureJ3
roc_sadness      #FigureJ4
roc_joy          #FigureJ5
roc_enthusiasm   #FigureJ6
roc_pride        #FigureJ7
roc_hope         #FigureJ8



#### Appendix K #######################################################
##### Load Data #################################################################
load("./results2.RData")


##### SEED ####################################################################

set.seed(1111)

##### Table K1 #################################################################
# In this exercise we take the 5000 sentences from the second crowd-coded dataset 
# which have been coded by 10 individuals each. 
# Then we draw on random subsamples from these 5000 sentences to estimate F1 scores 
# as a function of crowd coders per sentence. 
# To do so, we first create a helper function that randomly selects subsamples based on n (numbers of coders)

get_f1_anger <- function(data, number){
  # select sentences that have been coded by 10 crowd coders
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE) 
  
  new_df$h_anger <- NULL
  # count how many times each sentences has been coded as "Ärger" (anger)
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_anger_ind <- stringi::stri_count(new_df$answer, fixed = "Ärger")
  # create aggregated count of "anger" coding per sentence
  x <- aggregate(new_df$h_anger_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_anger"
  # create new binary variables (human judgement)
  new_df$hf_anger <- 0
  new_df$hf_anger[new_df$h_anger>0] <- 1
  # create new binary variables (ed8 dictionary)
  new_df$df_anger <- 0
  new_df$df_anger[new_df$anger.norm>0] <- 1
  # delete uncodables
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  # calculate scores for ed8 dictionary
  predict <- new_df2$df_anger
  true <- new_df2$hf_anger
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  # calculate scores for word embedding approach
  predict <- new_df2$wb.anger
  true <- new_df2$hf_anger
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  # calculate scores for ELECTRA approach
  predict <- as.numeric(new_df2$el.anger)
  true <- new_df2$hf_anger
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  #create one data frame
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}

# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores (recall, precision, F1) for each approach
tableK1 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_anger(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  anger_ed8_prec <- mean(test2$precision1)
  anger_ed8_recall <- mean(test2$recall1)
  anger_ed8_fscore <- mean(test2$Fmeasure1)
  anger_wb_prec <- mean(test2$precision3)
  anger_wb_recall <- mean(test2$recall3)
  anger_wb_fscore <- mean(test2$Fmeasure3)
  anger_elek_prec <- mean(test2$precision5)
  anger_elek_recall <- mean(test2$recall5)
  anger_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, anger_ed8_prec, anger_ed8_recall, anger_ed8_fscore, 
                      anger_wb_prec, anger_wb_recall, anger_wb_fscore, 
                      anger_elek_prec, anger_elek_recall,
                      anger_elek_fscore)
  tableK1 <- rbind(tableK1, test3)
}

print(tableK1)

# Exporting table to txt
stargazer(tableK1, summary = FALSE, out = "./tables/tableK1.txt", digits = 2)



##### Table K2 #################################################################
#### FEAR

get_f1_fear <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_fear <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_fear_ind <- stringi::stri_count(new_df$answer, fixed = "Angst")
  
  x <- aggregate(new_df$h_fear_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_fear"
  
  new_df$hf_fear <- 0
  new_df$hf_fear[new_df$h_fear>0] <- 1
  
  new_df$df_fear <- 0
  new_df$df_fear[new_df$fear.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_fear
  true <- new_df2$hf_fear
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.fear
  true <- new_df2$hf_fear
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.fear)
  true <- new_df2$hf_fear
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}

# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK2 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_fear(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  fear_ed8_prec <- mean(test2$precision1)
  fear_ed8_recall <- mean(test2$recall1)
  fear_ed8_fscore <- mean(test2$Fmeasure1)
  fear_wb_prec <- mean(test2$precision3)
  fear_wb_recall <- mean(test2$recall3)
  fear_wb_fscore <- mean(test2$Fmeasure3)
  fear_elek_prec <- mean(test2$precision5)
  fear_elek_recall <- mean(test2$recall5)
  fear_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, fear_ed8_prec, fear_ed8_recall, fear_ed8_fscore, 
                      fear_wb_prec, fear_wb_recall, fear_wb_fscore, 
                      fear_elek_prec, fear_elek_recall,
                      fear_elek_fscore)
  tableK2 <- rbind(tableK2, test3)
}

print(tableK2)

# Exporting table to txt
stargazer(tableK2, summary = FALSE, out = "./tables/tableK2.txt", digits = 2)


##### Table K3 #################################################################
#### DISGUST
get_f1_disgust <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_disgust <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_disgust_ind <- stringi::stri_count(new_df$answer, fixed = "Ekel")
  
  x <- aggregate(new_df$h_disgust_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_disgust"
  
  new_df$hf_disgust <- 0
  new_df$hf_disgust[new_df$h_disgust>0] <- 1
  
  new_df$df_disgust <- 0
  new_df$df_disgust[new_df$disgust.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_disgust
  true <- new_df2$hf_disgust
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.disgust
  true <- new_df2$hf_disgust
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.disgust)
  true <- new_df2$hf_disgust
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}

# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK3 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_disgust(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  disgust_ed8_prec <- mean(test2$precision1)
  disgust_ed8_recall <- mean(test2$recall1)
  disgust_ed8_fscore <- mean(test2$Fmeasure1)
  disgust_wb_prec <- mean(test2$precision3)
  disgust_wb_recall <- mean(test2$recall3)
  disgust_wb_fscore <- mean(test2$Fmeasure3)
  disgust_elek_prec <- mean(test2$precision5)
  disgust_elek_recall <- mean(test2$recall5)
  disgust_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, disgust_ed8_prec, disgust_ed8_recall, disgust_ed8_fscore, 
                      disgust_wb_prec, disgust_wb_recall, disgust_wb_fscore, 
                      disgust_elek_prec, disgust_elek_recall,
                      disgust_elek_fscore)
  tableK3 <- rbind(tableK3, test3)
}

print(tableK3)

# Exporting table to txt
stargazer(tableK3, summary = FALSE, out = "./tables/tableK3.txt", digits = 2)


##### Table K4 #################################################################
#### SADNESS


get_f1_sadness <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_sadness <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_sadness_ind <- stringi::stri_count(new_df$answer, fixed = "Traurigkeit")
  
  x <- aggregate(new_df$h_sadness_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_sadness"
  
  new_df$hf_sadness <- 0
  new_df$hf_sadness[new_df$h_sadness>0] <- 1
  
  new_df$df_sadness <- 0
  new_df$df_sadness[new_df$sadness.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_sadness
  true <- new_df2$hf_sadness
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.sadness
  true <- new_df2$hf_sadness
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.sadness)
  true <- new_df2$hf_sadness
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}


# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK4 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_sadness(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  sadness_ed8_prec <- mean(test2$precision1)
  sadness_ed8_recall <- mean(test2$recall1)
  sadness_ed8_fscore <- mean(test2$Fmeasure1)
  sadness_wb_prec <- mean(test2$precision3)
  sadness_wb_recall <- mean(test2$recall3)
  sadness_wb_fscore <- mean(test2$Fmeasure3)
  sadness_elek_prec <- mean(test2$precision5)
  sadness_elek_recall <- mean(test2$recall5)
  sadness_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, sadness_ed8_prec, sadness_ed8_recall, sadness_ed8_fscore, 
                      sadness_wb_prec, sadness_wb_recall, sadness_wb_fscore, 
                      sadness_elek_prec, sadness_elek_recall,
                      sadness_elek_fscore)
  tableK4 <- rbind(tableK4, test3)
}

print(tableK4)

# Exporting table to txt
stargazer(tableK4, summary = FALSE, out = "./tables/tableK4.txt", digits = 2)


##### Table K5 #################################################################
#### JOY

get_f1_joy <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_joy <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_joy_ind <- stringi::stri_count(new_df$answer, fixed = "Freude")
  
  x <- aggregate(new_df$h_joy_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_joy"
  
  new_df$hf_joy <- 0
  new_df$hf_joy[new_df$h_joy>0] <- 1
  
  new_df$df_joy <- 0
  new_df$df_joy[new_df$joy.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_joy
  true <- new_df2$hf_joy
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.joy
  true <- new_df2$hf_joy
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.joy)
  true <- new_df2$hf_joy
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}


# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK5 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_joy(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  joy_ed8_prec <- mean(test2$precision1)
  joy_ed8_recall <- mean(test2$recall1)
  joy_ed8_fscore <- mean(test2$Fmeasure1)
  joy_wb_prec <- mean(test2$precision3)
  joy_wb_recall <- mean(test2$recall3)
  joy_wb_fscore <- mean(test2$Fmeasure3)
  joy_elek_prec <- mean(test2$precision5)
  joy_elek_recall <- mean(test2$recall5)
  joy_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, joy_ed8_prec, joy_ed8_recall, joy_ed8_fscore, 
                      joy_wb_prec, joy_wb_recall, joy_wb_fscore, 
                      joy_elek_prec, joy_elek_recall,
                      joy_elek_fscore)
  tableK5 <- rbind(tableK5, test3)
}

print(tableK5)

# Exporting table to txt
stargazer(tableK5, summary = FALSE, out = "./tables/tableK5.txt", digits = 2)

##### Table K6 #################################################################
#### ENTHUSIASM

get_f1_enthusiasm <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_enthusiasm <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_enthusiasm_ind <- stringi::stri_count(new_df$answer, fixed = "Enthusiasmus")
  
  x <- aggregate(new_df$h_enthusiasm_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_enthusiasm"
  
  new_df$hf_enthusiasm <- 0
  new_df$hf_enthusiasm[new_df$h_enthusiasm>0] <- 1
  
  new_df$df_enthusiasm <- 0
  new_df$df_enthusiasm[new_df$enthusiasm.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_enthusiasm
  true <- new_df2$hf_enthusiasm
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.enthusiasm
  true <- new_df2$hf_enthusiasm
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.enthusiasm)
  true <- new_df2$hf_enthusiasm
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}


# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK6 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_enthusiasm(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  enthusiasm_ed8_prec <- mean(test2$precision1)
  enthusiasm_ed8_recall <- mean(test2$recall1)
  enthusiasm_ed8_fscore <- mean(test2$Fmeasure1)
  enthusiasm_wb_prec <- mean(test2$precision3)
  enthusiasm_wb_recall <- mean(test2$recall3)
  enthusiasm_wb_fscore <- mean(test2$Fmeasure3)
  enthusiasm_elek_prec <- mean(test2$precision5)
  enthusiasm_elek_recall <- mean(test2$recall5)
  enthusiasm_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, enthusiasm_ed8_prec, enthusiasm_ed8_recall, enthusiasm_ed8_fscore, 
                      enthusiasm_wb_prec, enthusiasm_wb_recall, enthusiasm_wb_fscore, 
                      enthusiasm_elek_prec, enthusiasm_elek_recall,
                      enthusiasm_elek_fscore)
  tableK6 <- rbind(tableK6, test3)
}

print(tableK6)

# Exporting table to txt
stargazer(tableK6, summary = FALSE, out = "./tables/tableK6.txt", digits = 2)

##### Table K7 #################################################################
#### PRIDE

get_f1_pride <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_pride <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_pride_ind <- stringi::stri_count(new_df$answer, fixed = "Stolz")
  
  x <- aggregate(new_df$h_pride_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_pride"
  
  new_df$hf_pride <- 0
  new_df$hf_pride[new_df$h_pride>0] <- 1
  
  new_df$df_pride <- 0
  new_df$df_pride[new_df$pride.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_pride
  true <- new_df2$hf_pride
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.pride
  true <- new_df2$hf_pride
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.pride)
  true <- new_df2$hf_pride
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}


# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK7 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_pride(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  pride_ed8_prec <- mean(test2$precision1)
  pride_ed8_recall <- mean(test2$recall1)
  pride_ed8_fscore <- mean(test2$Fmeasure1)
  pride_wb_prec <- mean(test2$precision3)
  pride_wb_recall <- mean(test2$recall3)
  pride_wb_fscore <- mean(test2$Fmeasure3)
  pride_elek_prec <- mean(test2$precision5)
  pride_elek_recall <- mean(test2$recall5)
  pride_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, pride_ed8_prec, pride_ed8_recall, pride_ed8_fscore, 
                      pride_wb_prec, pride_wb_recall, pride_wb_fscore, 
                      pride_elek_prec, pride_elek_recall,
                      pride_elek_fscore)
  tableK7 <- rbind(tableK7, test3)
}

print(tableK7)

# Exporting table to txt
stargazer(tableK7, summary = FALSE, out = "./tables/tableK7.txt", digits = 2)

##### Table K8 #################################################################
#### HOPE

get_f1_hope <- function(data, number){
  new_df <- data[data$n_coders==10,] %>% group_by(TextID) %>% sample_n(number, replace = TRUE)
  
  new_df$h_hope <- NULL
  
  new_df$answer <- paste(new_df$Answer.1, new_df$Answer.2, new_df$Answer.3, new_df$Answer.4, sep = " ")
  new_df$h_hope_ind <- stringi::stri_count(new_df$answer, fixed = "Hoffnung")
  
  x <- aggregate(new_df$h_hope_ind, by=list(new_df$TextID), sum)
  colnames(x)[1] <- "TextID"
  new_df <- full_join(new_df, x, by = "TextID")
  colnames(new_df)[112] <- "h_hope"
  
  new_df$hf_hope <- 0
  new_df$hf_hope[new_df$h_hope>0] <- 1
  
  new_df$df_hope <- 0
  new_df$df_hope[new_df$hope.norm>0] <- 1
  
  new_df2 <- new_df[!duplicated(new_df$TextID),]
  new_df2 <- new_df2[new_df2$h_uncodable<2,]
  
  predict <- new_df2$df_hope
  true <- new_df2$hf_hope
  
  retrieved <- sum(predict)
  precision1 <- sum(predict & true) / retrieved
  recall1 <- sum(predict & true) / sum(true)
  Fmeasure1 <- 2 * precision1 * recall1 / (precision1 + recall1)
  
  predict <- new_df2$wb.hope
  true <- new_df2$hf_hope
  
  retrieved <- sum(predict)
  precision3 <- sum(predict & true) / retrieved
  recall3 <- sum(predict & true) / sum(true)
  Fmeasure3 <- 2 * precision3 * recall3 / (precision3 + recall3)
  
  predict <- as.numeric(new_df2$el.hope)
  true <- new_df2$hf_hope
  
  retrieved <- sum(predict)
  precision5 <- sum(predict & true) / retrieved
  recall5 <- sum(predict & true) / sum(true)
  Fmeasure5 <- 2 * precision5 * recall5 / (precision5 + recall5)
  
  
  Fmeasure <- data.frame(precision1, recall1, Fmeasure1, precision3, recall3, Fmeasure3, precision5, recall5, Fmeasure5)
  return(Fmeasure)
}


# We now bootstrap 1000 sets of subsamples with replacement for each n ranging from n = 1 to n = 10
# and calculate mean scores for each approach
tableK8 <- NULL
for (j in 1:10){
  test <- lapply(seq_len(1000), function(x) get_f1_hope(results2, j))
  test2 <- bind_rows(test, .id = "replication")
  hope_ed8_prec <- mean(test2$precision1)
  hope_ed8_recall <- mean(test2$recall1)
  hope_ed8_fscore <- mean(test2$Fmeasure1)
  hope_wb_prec <- mean(test2$precision3)
  hope_wb_recall <- mean(test2$recall3)
  hope_wb_fscore <- mean(test2$Fmeasure3)
  hope_elek_prec <- mean(test2$precision5)
  hope_elek_recall <- mean(test2$recall5)
  hope_elek_fscore <- mean(test2$Fmeasure5)
  n_coders <- j
  test3 <- data.frame(n_coders, hope_ed8_prec, hope_ed8_recall, hope_ed8_fscore, 
                      hope_wb_prec, hope_wb_recall, hope_wb_fscore, 
                      hope_elek_prec, hope_elek_recall,
                      hope_elek_fscore)
  tableK8 <- rbind(tableK8, test3)
}

print(tableK8)

# Exporting table to txt
stargazer(tableK8, summary = FALSE, out = "./tables/tableK8.txt", digits = 2)

##### Figure K1 ######################################################################
# Now, we use the data from above (Tabke K1 - K8) to display the results graphically
# We first reshape the data frames into long format, and then use ggplot2 to create the graphs
# for each emotion

pd <- position_dodge(0.1) # move them .1 to the left and right

anger_long <- reshape(tableK1, 
                      direction = "long",
                      varying = list(names(tableK1)[c(4,7,10)]),
                      v.names = "F1",
                      idvar = c("n_coders"),
                      timevar = "Approach")

anger_long <- reshape2::melt(tableK1, id.vars=c("n_coders"), measure.vars=c("anger_ed8_fscore", "anger_wb_fscore", "anger_elek_fscore" ))

anger <- ggplot(anger_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='anger') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))

fear_long <- reshape(tableK2, 
                     direction = "long",
                     varying = list(names(tableK2)[c(4,7,10)]),
                     v.names = "F1",
                     idvar = c("n_coders"),
                     timevar = "Approach")

fear_long <- reshape2::melt(tableK2, id.vars=c("n_coders"), measure.vars=c("fear_ed8_fscore", "fear_wb_fscore", "fear_elek_fscore" ))

fear <- ggplot(fear_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='fear') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))


disgust_long <- reshape(tableK3, 
                        direction = "long",
                        varying = list(names(tableK3)[c(4,7,10)]),
                        v.names = "F1",
                        idvar = c("n_coders"),
                        timevar = "Approach")

disgust_long <- reshape2::melt(tableK3, id.vars=c("n_coders"), measure.vars=c("disgust_ed8_fscore", "disgust_wb_fscore", "disgust_elek_fscore" ))

disgust <- ggplot(disgust_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='disgust') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))

sadness_long <- reshape(tableK4, 
                        direction = "long",
                        varying = list(names(tableK4)[c(4,7,10)]),
                        v.names = "F1",
                        idvar = c("n_coders"),
                        timevar = "Approach")

sadness_long <- reshape2::melt(tableK4, id.vars=c("n_coders"), measure.vars=c("sadness_ed8_fscore", "sadness_wb_fscore", "sadness_elek_fscore" ))

sadness <- ggplot(sadness_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='sadness') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))


figureK1 <- ggarrange(anger, fear, disgust, sadness,
                      ncol = 2, nrow = 2)

figureK1 

ggsave("./figures/figureK1.pdf", scale = 2)



##### Figure K2 ######################################################################

joy_long <- reshape(tableK5, 
                    direction = "long",
                    varying = list(names(tableK5)[c(4,7,10)]),
                    v.names = "F1",
                    idvar = c("n_coders"),
                    timevar = "Approach")

joy_long <- reshape2::melt(tableK5, id.vars=c("n_coders"), measure.vars=c("joy_ed8_fscore", "joy_wb_fscore", "joy_elek_fscore" ))

joy <- ggplot(joy_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='joy') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))


enthusiasm_long <- reshape(tableK6, 
                           direction = "long",
                           varying = list(names(tableK6)[c(4,7,10)]),
                           v.names = "F1",
                           idvar = c("n_coders"),
                           timevar = "Approach")

enthusiasm_long <- reshape2::melt(tableK6, id.vars=c("n_coders"), measure.vars=c("enthusiasm_ed8_fscore", "enthusiasm_wb_fscore", "enthusiasm_elek_fscore" ))

enthusiasm <- ggplot(enthusiasm_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='enthusiasm') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))

pride_long <- reshape(tableK7, 
                      direction = "long",
                      varying = list(names(tableK7)[c(4,7,10)]),
                      v.names = "F1",
                      idvar = c("n_coders"),
                      timevar = "Approach")

pride_long <- reshape2::melt(tableK7, id.vars=c("n_coders"), measure.vars=c("pride_ed8_fscore", "pride_wb_fscore", "pride_elek_fscore" ))

pride <- ggplot(pride_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='pride') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))


hope_long <- reshape(tableK8, 
                     direction = "long",
                     varying = list(names(tableK8)[c(4,7,10)]),
                     v.names = "F1",
                     idvar = c("n_coders"),
                     timevar = "Approach")

hope_long <- reshape2::melt(tableK8, id.vars=c("n_coders"), measure.vars=c("hope_ed8_fscore", "hope_wb_fscore", "hope_elek_fscore" ))

hope <- ggplot(hope_long, aes(x=as.factor(n_coders), y=value, colour=variable, group=variable)) +
  geom_line(linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() +
  xlab("Number of coders") + 
  ylab("F1 score") +
  geom_vline(xintercept = 5, linetype= "dashed", color="grey") +
  labs(color='Approach', title='hope') +
  scale_color_manual(labels=c("ed8 Dictionary","Word Embeddings","Transformer-Based"),
                     values=c("#F8766D", "#00BFC4", "#7CAE00")) +
  scale_x_discrete(limits=factor(c(1:10)))


figureK2 <- ggarrange(joy, enthusiasm, pride, hope,
                      ncol = 2, nrow = 2)

figureK2 

ggsave("./figures/figureK2.pdf", scale = 2)


#### Appendix L ##########################################################
##### Load Data #################################################################
load("./data1_prepared.RData")



##### Create variabels including stopwords #####################################
# To check, whether the removal of stopwords in the calculation of the emotional scores
# biased the results, we first have to create new scores including stopwords 

data1_prepared$anger.norm_st <- data1_prepared$anger / data1_prepared$terms.raw
data1_prepared$fear.norm_st <- data1_prepared$fear / data1_prepared$terms.raw
data1_prepared$disgust.norm_st <- data1_prepared$disgust / data1_prepared$terms.raw
data1_prepared$sadness.norm_st <- data1_prepared$sadness / data1_prepared$terms.raw
data1_prepared$joy.norm_st <- data1_prepared$joy / data1_prepared$terms.raw
data1_prepared$enthusiasm.norm_st <- data1_prepared$enthusiasm / data1_prepared$terms.raw
data1_prepared$pride.norm_st <- data1_prepared$pride / data1_prepared$terms.raw
data1_prepared$hope.norm_st <- data1_prepared$hope / data1_prepared$terms.raw

##### Create categorical variables ############################################
# Then, we create categorical variables based on human judgement
# We assign "clearly emotional" to sentences which have been coded as 'emotional' by 5 or 4 human coders
# We assign "emotional" to sentences which have been coded as 'emotional' by 3 or 2 human coders
# We assign "not/slightly emotion" to sentences which have been coded as 'emotional' by 0 or 1 human coders

## Categorical
data1_prepared$h_anger4 <- NULL
data1_prepared$h_anger4[data1_prepared$h_anger==5 | data1_prepared$h_anger == 4] <- "clearly emotional"
data1_prepared$h_anger4[data1_prepared$h_anger==3 | data1_prepared$h_anger == 2] <- "emotional"
data1_prepared$h_anger4[data1_prepared$h_anger == 0 | data1_prepared$h_anger == 1] <- "not/slightly"

data1_prepared$h_fear4 <- NULL
data1_prepared$h_fear4[data1_prepared$h_fear==5 | data1_prepared$h_fear == 4] <- "clearly emotional"
data1_prepared$h_fear4[data1_prepared$h_fear==3 | data1_prepared$h_fear == 2] <- "emotional"
data1_prepared$h_fear4[data1_prepared$h_fear == 0 | data1_prepared$h_fear == 1] <- "not/slightly"

data1_prepared$h_disgust4 <- NULL
data1_prepared$h_disgust4[data1_prepared$h_disgust==5 | data1_prepared$h_disgust == 4] <- "clearly emotional"
data1_prepared$h_disgust4[data1_prepared$h_disgust==3 | data1_prepared$h_disgust == 2] <- "emotional"
data1_prepared$h_disgust4[data1_prepared$h_disgust == 0 | data1_prepared$h_disgust == 1] <- "not/slightly"

data1_prepared$h_sadness4 <- NULL
data1_prepared$h_sadness4[data1_prepared$h_sadness==5 | data1_prepared$h_sadness == 4] <- "clearly emotional"
data1_prepared$h_sadness4[data1_prepared$h_sadness==3 | data1_prepared$h_sadness == 2] <- "emotional"
data1_prepared$h_sadness4[data1_prepared$h_sadness == 0 | data1_prepared$h_sadness == 1] <- "not/slightly"

data1_prepared$h_joy4 <- NULL
data1_prepared$h_joy4[data1_prepared$h_joy==5 | data1_prepared$h_joy == 4] <- "clearly emotional"
data1_prepared$h_joy4[data1_prepared$h_joy==3 | data1_prepared$h_joy == 2] <- "emotional"
data1_prepared$h_joy4[data1_prepared$h_joy == 0 | data1_prepared$h_joy == 1] <- "not/slightly"

data1_prepared$h_enthusiasm4 <- NULL
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm==5 | data1_prepared$h_enthusiasm == 4] <- "clearly emotional"
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm==3 | data1_prepared$h_enthusiasm == 2] <- "emotional"
data1_prepared$h_enthusiasm4[data1_prepared$h_enthusiasm == 0 | data1_prepared$h_enthusiasm == 1] <- "not/slightly"

data1_prepared$h_pride4 <- NULL
data1_prepared$h_pride4[data1_prepared$h_pride==5 | data1_prepared$h_pride == 4] <- "clearly emotional"
data1_prepared$h_pride4[data1_prepared$h_pride==3 | data1_prepared$h_pride == 2] <- "emotional"
data1_prepared$h_pride4[data1_prepared$h_pride == 0 | data1_prepared$h_pride == 1] <- "not/slightly"

data1_prepared$h_hope4 <- NULL
data1_prepared$h_hope4[data1_prepared$h_hope==5 | data1_prepared$h_hope == 4] <- "clearly emotional"
data1_prepared$h_hope4[data1_prepared$h_hope==3 | data1_prepared$h_hope == 2] <- "emotional"
data1_prepared$h_hope4[data1_prepared$h_hope == 0 | data1_prepared$h_hope == 1] <- "not/slightly"


#Then we order the variable levels so the graphs go from the lowest to the highest value
data1_prepared$h_anger4 <- ordered(data1_prepared$h_anger4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_fear4 <- ordered(data1_prepared$h_fear4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_disgust4 <- ordered(data1_prepared$h_disgust4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_sadness4 <- ordered(data1_prepared$h_sadness4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_joy4 <- ordered(data1_prepared$h_joy4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_enthusiasm4 <- ordered(data1_prepared$h_enthusiasm4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_pride4 <- ordered(data1_prepared$h_pride4,levels = c("not/slightly", "emotional", "clearly emotional"))
data1_prepared$h_hope4 <- ordered(data1_prepared$h_hope4,levels = c("not/slightly", "emotional", "clearly emotional"))





##### Figure L1 ##################################################################
# Then, we display the two scores graphically against human judgement
# To do so, we first create a dataframe including both scores by categories of human judgement

pd <- position_dodge(0.1) # move them .05 to the left and right

### Anger
data1_prepared_normal <- select(data1_prepared, anger.norm, h_anger4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "anger.norm"
data1_prepared_st <- select(data1_prepared, anger.norm_st, h_anger4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "anger.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$anger.norm <- as.numeric(data1_prepared2$anger.norm)

# we produce count, mean, standard deviation, standard error of the mean, and confidence interval (default 95%)
df2 <- summarySE(data1_prepared2, measurevar="anger.norm", groupvars=c("h_anger4", "dic"), na.rm=TRUE)

# Plot
anger <- ggplot(df2, aes(x=h_anger4, y=anger.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=anger.norm-ci, ymax=anger.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("anger \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Fear
data1_prepared_normal <- select(data1_prepared, fear.norm, h_fear4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "fear.norm"
data1_prepared_st <- select(data1_prepared, fear.norm_st, h_fear4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "fear.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$fear.norm <- as.numeric(data1_prepared2$fear.norm)

df2 <- summarySE(data1_prepared2, measurevar="fear.norm", groupvars=c("h_fear4", "dic"), na.rm=TRUE)

fear <- ggplot(df2, aes(x=h_fear4, y=fear.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=fear.norm-ci, ymax=fear.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("fear \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Disgust
data1_prepared_normal <- select(data1_prepared, disgust.norm, h_disgust4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "disgust.norm"
data1_prepared_st <- select(data1_prepared, disgust.norm_st, h_disgust4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "disgust.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$disgust.norm <- as.numeric(data1_prepared2$disgust.norm)

df2 <- summarySE(data1_prepared2, measurevar="disgust.norm", groupvars=c("h_disgust4", "dic"), na.rm=TRUE)

disgust <- ggplot(df2, aes(x=h_disgust4, y=disgust.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=disgust.norm-ci, ymax=disgust.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("disgust \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Sadness
data1_prepared_normal <- select(data1_prepared, sadness.norm, h_sadness4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "sadness.norm"
data1_prepared_st <- select(data1_prepared, sadness.norm_st, h_sadness4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "sadness.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$sadness.norm <- as.numeric(data1_prepared2$sadness.norm)

df2 <- summarySE(data1_prepared2, measurevar="sadness.norm", groupvars=c("h_sadness4", "dic"), na.rm=TRUE)


sadness <- ggplot(df2, aes(x=h_sadness4, y=sadness.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=sadness.norm-ci, ymax=sadness.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("sadness \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))


figureL1 <- ggarrange(anger, fear, disgust, sadness,
                      ncol = 2, nrow = 2)

figureL1 

ggsave("./figures/figureL1.pdf", scale = 2)




##### Figure L2 #####################################################################
### Joy
data1_prepared_normal <- select(data1_prepared, joy.norm, h_joy4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "joy.norm"
data1_prepared_st <- select(data1_prepared, joy.norm_st, h_joy4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "joy.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$joy.norm <- as.numeric(data1_prepared2$joy.norm)

df2 <- summarySE(data1_prepared2, measurevar="joy.norm", groupvars=c("h_joy4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

joy <- ggplot(df2, aes(x=h_joy4, y=joy.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=joy.norm-ci, ymax=joy.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("joy \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Enthusiasm
data1_prepared_normal <- select(data1_prepared, enthusiasm.norm, h_enthusiasm4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "enthusiasm.norm"
data1_prepared_st <- select(data1_prepared, enthusiasm.norm_st, h_enthusiasm4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "enthusiasm.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$enthusiasm.norm <- as.numeric(data1_prepared2$enthusiasm.norm)

df2 <- summarySE(data1_prepared2, measurevar="enthusiasm.norm", groupvars=c("h_enthusiasm4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

enthusiasm <- ggplot(df2, aes(x=h_enthusiasm4, y=enthusiasm.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=enthusiasm.norm-ci, ymax=enthusiasm.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("enthusiasm \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Pride
data1_prepared_normal <- select(data1_prepared, pride.norm, h_pride4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "pride.norm"
data1_prepared_st <- select(data1_prepared, pride.norm_st, h_pride4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "pride.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$pride.norm <- as.numeric(data1_prepared2$pride.norm)

df2 <- summarySE(data1_prepared2, measurevar="pride.norm", groupvars=c("h_pride4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

pride <- ggplot(df2, aes(x=h_pride4, y=pride.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=pride.norm-ci, ymax=pride.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("pride \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))

### Hope
data1_prepared_normal <- select(data1_prepared, hope.norm, h_hope4)
data1_prepared_normal$dic <- "Stopwords excluded"
colnames(data1_prepared_normal)[1] <- "hope.norm"
data1_prepared_st <- select(data1_prepared, hope.norm_st, h_hope4)
data1_prepared_st$dic <- "Stopwords included"
colnames(data1_prepared_st)[1] <- "hope.norm"
data1_prepared2 <- rbind(data1_prepared_normal, data1_prepared_st)
data1_prepared2$hope.norm <- as.numeric(data1_prepared2$hope.norm)

df2 <- summarySE(data1_prepared2, measurevar="hope.norm", groupvars=c("h_hope4", "dic"), na.rm=TRUE)

pd <- position_dodge(0.1) # move them .05 to the left and right

hope <- ggplot(df2, aes(x=h_hope4, y=hope.norm, colour=dic, group=dic)) +
  geom_errorbar(aes(ymin=hope.norm-ci, ymax=hope.norm+ci), width=.1, position=pd) +
  geom_line(position=pd, linetype="dashed") +
  geom_point(position=pd) +
  theme_bw() + 
  xlab("Human coders") + 
  ylab("hope \n(Normalized emotional scores)") +
  labs(color='Type') +
  scale_color_manual(values=c("#F8766D", "#00BFC4", "#7CAE00"))


figureL2 <- ggarrange(joy, enthusiasm, pride, hope,
                      ncol = 2, nrow = 2)

figureL2

ggsave("./figures/figureL2.pdf", scale = 2)

##### Table L1 ######################################################################
# Lastly, we produce a correlation table between emotions (as judged by humans) and stopwords

cor_data <- data1_prepared[, c(21, 54, 58, 62, 65, 69, 72, 74, 76)]

res <- as.data.frame(cor(cor_data))
round(res, 3)

# Exporting table to txt
stargazer(res, summary = FALSE, out = "./tables/tableL1.txt", digits = 3)



#### Appendix M ##########################################################
##### Load Data #################################################################
load("./case_sample.RData") # Press releases from Germany

##### Apply ELECTRA Classifier ####################################################

# To apply the Electra model, you first need to turn the R dataframe into a csv that can be used
# in the Python script
write.csv(case_sample,'./electra/electra_appendixm.csv')

# Then you can read the csv file in into Python, following the code in the Python script 05_apply_transformer_based_model.ipynb 

# After using the Python script, read the results file back into the R environment
electra_results <- read.csv('./electra/electra_results_appendixm.csv', header = TRUE)

# Change column names before binding it to the main dataframe
colnames(electra_results)[3] <- "el.anger"
colnames(electra_results)[4] <- "el.fear"
colnames(electra_results)[5] <- "el.disgust"
colnames(electra_results)[6] <- "el.sadness"
colnames(electra_results)[7] <- "el.joy"
colnames(electra_results)[8] <- "el.enthusiasm"
colnames(electra_results)[9] <- "el.pride"
colnames(electra_results)[10] <- "el.hope"

# Delete columns unnecessary columns
electra_results$X <- electra_results$text <- NULL

# Add results to the main dataframe
case_sample <- cbind(case_sample, electra_results)


##### Figure M1 ##################################################################
# Create "position" variable: including factors for government, opposition, and radical parties

case_sample$position[case_sample$party=="AfD" | case_sample$party=="Die Linke"] <- "Radical"
case_sample$position[case_sample$party=="FDP GER" | case_sample$party=="Grüne GER"] <- "Opposition"
case_sample$position[case_sample$party=="CDU" | case_sample$party=="CSU" | case_sample$party=="SPD"] <- "Government"

# Create mean scores for each emotion by party type
position_df1 <- summarySE(case_sample, measurevar="el.anger", groupvars=c("position"))
colnames(position_df1)[3] <- "value"
position_df1$emotion <- "anger"
position_df2 <- summarySE(case_sample, measurevar="el.fear", groupvars=c("position"))
colnames(position_df2)[3] <- "value"
position_df2$emotion <- "fear"
position_df3 <- summarySE(case_sample, measurevar="el.disgust", groupvars=c("position"))
colnames(position_df3)[3] <- "value"
position_df3$emotion <- "disgust"
position_df4 <- summarySE(case_sample, measurevar="el.sadness", groupvars=c("position"))
colnames(position_df4)[3] <- "value"
position_df4$emotion <- "sadness"
position_df <- rbind(position_df1, position_df2, position_df3, position_df4)

figureM1 <- ggplot(position_df, aes(x=as.factor(emotion), y=value, fill=as.factor(position))) + 
  geom_bar(position=position_dodge(), stat="identity") +
  geom_errorbar(aes(ymin=value-ci, ymax=value+ci),
                width=.2,                    # Width of the error bars
                position=position_dodge(.9)) +
  xlab("Emotion")+ylab("Mean Score") +
  theme_bw() +
  labs(fill='Party Group', title='Negative Emotions') +
  scale_x_discrete(limits=c("anger", "fear", "disgust", "sadness")) +
  scale_fill_manual(labels=c("Government","Opposition","Radical"),
                    values=c("#F8766D", "#00BFC4", "#7CAE00"))

figureM1

ggsave("./figures/figureM1.pdf", scale = 2)

##### Figure M2 ####################################################################

position_df1 <- summarySE(case_sample, measurevar="el.joy", groupvars=c("position"))
colnames(position_df1)[3] <- "value"
position_df1$emotion <- "joy"
position_df2 <- summarySE(case_sample, measurevar="el.enthusiasm", groupvars=c("position"))
colnames(position_df2)[3] <- "value"
position_df2$emotion <- "enthusiasm"
position_df3 <- summarySE(case_sample, measurevar="el.pride", groupvars=c("position"))
colnames(position_df3)[3] <- "value"
position_df3$emotion <- "pride"
position_df4 <- summarySE(case_sample, measurevar="el.hope", groupvars=c("position"))
colnames(position_df4)[3] <- "value"
position_df4$emotion <- "hope"
position_df <- rbind(position_df1, position_df2, position_df3, position_df4)

figureM2 <- ggplot(position_df, aes(x=as.factor(emotion), y=value, fill=as.factor(position))) + 
  geom_bar(position=position_dodge(), stat="identity") +
  geom_errorbar(aes(ymin=value-ci, ymax=value+ci),
                width=.2,                    # Width of the error bars
                position=position_dodge(.9)) +
  xlab("Emotion")+ylab("Mean Score") +
  theme_bw() +
  labs(fill='Party Group', title='Positive Emotions') +
  scale_x_discrete(limits=c("joy", "enthusiasm", "pride", "hope")) +
  scale_fill_manual(labels=c("Government","Opposition","Radical"),
                    values=c("#F8766D", "#00BFC4", "#7CAE00"))


figureM2

ggsave("./figures/figureM2.pdf", scale = 2)


##### Figure M3 ################################################################
# Create campaign variable: 3 months before election
case_sample$campaign <- 0
case_sample$campaign[case_sample$date>"2017-06-23" & case_sample$date<"2017-09-24"] <- 1

# Run regression analysis for negative emotions
model1 <- lm(el.anger ~ campaign, data = case_sample)
model2 <- lm(el.fear ~ campaign, data = case_sample)
model3 <- lm(el.disgust ~ campaign, data = case_sample)
model4 <- lm(el.sadness ~ campaign, data = case_sample)

# Display results in coefficient plots
figureM3 <- dwplot(list(model1, model2, model3, model4), show_intercept = FALSE) +
  geom_vline(xintercept = 0, colour = "grey60", linetype = 2) +
  scale_color_manual(labels = c("Anger", "Fear", "Disgust", "Sadness"), name = "Emotion",
                     values = c("#F8766D", "#00BFC4", "#7CAE00", "royalblue1")) +
  theme_bw() 

figureM3 

ggsave("./figures/figureM3.pdf", scale = 2)

##### Figure M4 ################################################################

# Run regression analysis for positive emotions
model5 <- lm(el.joy ~ campaign, data = case_sample)
model6 <- lm(el.enthusiasm ~ campaign, data = case_sample)
model7 <- lm(el.pride ~ campaign, data = case_sample)
model8 <- lm(el.hope ~ campaign, data = case_sample)

# Display results in coefficient plots
figureM4 <- dwplot(list(model5, model6, model7, model8), show_intercept = FALSE) +
  geom_vline(xintercept = 0, colour = "grey60", linetype = 2) +
  scale_color_manual(labels = c("Joy", "Enthusiasm", "Pride", "Hope"), name = "Emotion",
                     values = c("#F8766D", "#00BFC4", "#7CAE00", "royalblue1")) +
  theme_bw()


figureM4

ggsave("./figures/figureM4.pdf", scale = 2)



####################################################################################
### END OF SCRIPT ###
####################################################################################


